#!/usr/bin/env python
# -*- coding: utf-8 -*-
import scrapy,re,logging,requests,json
from scrapy.selector import Selector
from jdall.items import *
from scrapy.http import Request
import urllib.request
from scrapy import Spider
import ssl

# from urllib.request import Request

key_words = ['book', 'e', 'channel', 'mvd', 'list']
base_dir = 'https://list.jd.com'
# price_url = 'https://p.3.cn/prices/mgets?skuIds=J_'
comment_url = 'https://club.jd.com/comment/productPageComments.action?productId=%s&score=0&sortType=5&page=%s&pageSize=10'
favourable_url = 'https://cd.jd.com/promotion/v2?skuId=%s&area=1_72_2799_0&shopId=%s&venderId=%s&cat=%s'
price_url = 'https://p.3.cn/prices/mgets?callback=jQuery4137689&type=1&area=1_72_4137_0&pdtk=&pduid=15284148123261481286051&pdpin=&pin=null&pdbp=0&skuIds=J_%s&ext=11100000&source=item-pc'


class JdSpider(scrapy.Spider):
    name = "jd"
    allowed_domains = ["jd.com"]
    start_urls = ['https://www.jd.com/allSort.aspx']

    logging.getLogger("requests").setLevel(logging.WARNING)

    def start_requests(self):
        yield Request(url=self.start_urls[0],callback=self.parse_category)



    def parse_category(self, response):
        """获取分类页"""
        try:
            selector = Selector(response)
            texts = selector.xpath('//div[@class = "category-item m"]/div[@class = "mc"]/div[@class = "items"]/dl/dd/a' ).extract()
            for text in texts:
                items = re.findall(r'<a href="(.*?)" target="_blank">(.*?)</a>',text)
                for item in items:
                    if item[0].split('.')[0][2:] in key_words:
                        if item[0].split('.')[0][2:] != 'list':
                            yield Request(url='https:' + item[0], callback=self.parse_category)
                        else:
                            categoriesItem = CategoriesItem()
                            categoriesItem['name'] =item[1]
                            categoriesItem['url'] = item[0][2:]
                            categoriesItem['id'] = item[0].split('=')[1].split('&')[0]
                            yield Request(url= 'https:'+item[0],callback=self.parse_list)
        except Exception as e:
            print(e)
    def parse_list(self,response):
        this_url = response.url
        urlpat = 'https://list.jd.com(.*?)'
        x = re.search(urlpat,this_url)
        meta = dict()

        if(x):

            meta['category'] = response.url.split('=')[1].split('&')[0]

            selector = Selector(response)
            texts = selector.xpath('//*[@id="plist"]/ul/li/div/div[@class = "p-img"]/a').extract()
            for text in texts:
                items = re.findall(r'<a target="_blank" href="(.*?)">',text)   #商品详情页链接
                yield Request(url='https:'+items[0],callback=self.parse_prod,meta=meta)

    ###提取下一页字段
            # next_list = response.xpath('//a[@class = "pn-next"]/@href').extract()
            # if(next_list):
            #     yield Request(url=base_dir+next_list[0],callback=self.parse_list)


    def parse_prod(self,response):
        # 详情页处理

        category = response.meta['category']
        ids = re.findall(r"venderId:(.*?),\s.*?shopId:'(.*?)',",response.text)

        if not ids:
            ids = re.findall(r"venderId:(.*?),\s.*?shopId:(.*?),", response.text)
        if (ids):
            venderID = ids[0][0]
            shopID = ids[0][1]


        #店铺信息
        shopItem =ShopItem()

        shopItem['shopID'] = shopID
        shopItem['venderID'] = venderID
        shopItem['url1'] = 'https://mall.jd.com/index-%s.html' %(shopID)
        try:
            shopItem['url2'] = 'https:' + response.xpath('//ul[@class="parameter2 p-parameter-list"]/li/a/@href').extract()[0]
        except:
            shopItem['url2'] = shopItem['url1']

        name = ''
        if shopID == '0':
            # name = '京东自营'
            pass
        else:
            try:
                name = response.xpath('//ul[@class="parameter2 p-parameter-list"]/li/a//text()').extract()[0]
            except:
                try:
                    name = response.xpath('//div[@class="name"]/a//text()').extract()[0].strip()
                except:
                    try:
                        name = response.xpath('//div[@class="shopName"]/strong/span/a//text()').extract()[0].strip()
                    except:
                        try:
                            name = response.xpath('//div[@class="seller-infor"]/a//text()').extract()[0].strip()
                        except Exception as e:
                            print(e)
                # name = u'京东自营'

        shopItem['name'] = name
        shopItem['id'] = name

        yield shopItem

        productsItem = ProductsItem()
        productsItem['shopID'] = shopID
        productsItem['category'] = category
        try:
            title = response.xpath('//div[@class = "sku-name"]/text()').extract()[0].replace(u'\xa0', '').strip()
        except :
            title = response.xpath('//div[@id="name"]/h1/text()').extract()[0]
        productsItem['name'] = title
        product_id = response.url.split('/')[-1][:-5]
        productsItem['url'] = response.url
        #商品描述
        desc = response.xpath('//ul[@class="parameter2 p-parameter-list"]//li/text()').extract()
        productsItem['description'] = ';'.join(i.strip() for i in desc)
        # print(productsItem['description'])
        #价格
        # response = requests.get(url=price_url+product_id)
        #
        # price_json = response.json()
        #
        #  = price_json[0]['m']
        # print(price_json)
        really_price_url = price_url %(product_id)
        price_texts = urllib.request.urlopen(really_price_url,context=ssl._create_unverified_context()).read().decode('utf-8',"ignore")
        originalPrice_pat = '"m":"(.*?)"'
        reallyPrice_pat = '"p":"(.*?)"'
        price_data_originalPrice=re.compile(originalPrice_pat).findall(price_texts)
        price_data_reallyPrice = re.compile(reallyPrice_pat).findall(price_texts)

        productsItem['reallyPrice'] = ''.join(price_data_reallyPrice)
        productsItem['originalPrice'] = ''.join(price_data_originalPrice)

        #优惠
        res_url = favourable_url % (product_id, shopID, venderID, category.replace(',', '%2c'))
        response = requests.get(res_url)
        fav_data = dict()
        try:
            fav_data = response.json()
        except:
            pass
        if (fav_data.get('skuCoupon')):
            desc1 = []

            for item in fav_data['skuCoupon']:
                start_time = item['beginTime']
                end_time = item['endTime']
                time_dec = item['timeDesc']
                fav_price = item['quota']
                fav_count = item['discount']
                fav_time = item['addDays']
                desc1.append(u'有效期%s至%s,满%s减%s' % (start_time, end_time, fav_price, fav_count))
            productsItem['favourableDesc1'] = ';'.join(desc1)
        if fav_data['prom'] and fav_data['prom']['pickOneTag']:
            desc2 = []
            for item in fav_data['prom']['pickOneTag']:
                desc2.append(item['content'])
            productsItem['favourableDesc1'] =','.join(desc2)

        data = dict()
        data['product_id'] = product_id
        yield productsItem

        yield Request(url=comment_url %(product_id ,'0'),callback=self.parse_comments,meta = data)


    def parse_comments(self,response):
        try:
            data = json.loads(response.text)

        except Exception as e:
            print(e)
            return  None

        product_id = response.meta['product_id']
        commentSummaryItem = CommentSummaryItem()
        commentSummary = data.get('productCommentSummary')
        commentSummaryItem['goodRateShow'] = commentSummary.get('goodRateShow')
        commentSummaryItem['poorRateShow'] = commentSummary.get('poorRateShow')
        commentSummaryItem['poorCountStr'] = commentSummary.get('poorCountStr')
        commentSummaryItem['averageScore'] = commentSummary.get('averageScore')
        commentSummaryItem['generalCountStr'] = commentSummary.get('generalCountStr')
        commentSummaryItem['showCount'] = commentSummary.get('showCount')
        commentSummaryItem['showCountStr'] = commentSummary.get('showCountStr')
        commentSummaryItem['goodCount'] = commentSummary.get('goodCount')
        commentSummaryItem['generalRate'] = commentSummary.get('generalRate')
        commentSummaryItem['generalCount'] = commentSummary.get('generalCount')
        commentSummaryItem['skuId'] = commentSummary.get('skuId')
        commentSummaryItem['goodCountStr'] = commentSummary.get('goodCountStr')
        commentSummaryItem['poorRate'] = commentSummary.get('poorRate')
        commentSummaryItem['afterCount'] = commentSummary.get('afterCount')
        commentSummaryItem['goodRateStyle'] = commentSummary.get('goodRateStyle')
        commentSummaryItem['poorCount'] = commentSummary.get('poorCount')
        commentSummaryItem['skuIds'] = commentSummary.get('skuIds')
        commentSummaryItem['poorRateStyle'] = commentSummary.get('poorRateStyle')
        commentSummaryItem['generalRateStyle'] = commentSummary.get('generalRateStyle')
        commentSummaryItem['commentCountStr'] = commentSummary.get('commentCountStr')
        commentSummaryItem['commentCount'] = commentSummary.get('commentCount')
        commentSummaryItem['productId'] = commentSummary.get('productId')  # 同ProductsItem的id相同
        commentSummaryItem['id'] = commentSummary.get('productId')
        commentSummaryItem['afterCountStr'] = commentSummary.get('afterCountStr')
        commentSummaryItem['goodRate'] = commentSummary.get('goodRate')
        commentSummaryItem['generalRateShow'] = commentSummary.get('generalRateShow')
        commentSummaryItem['jwotestProduct'] = data.get('jwotestProduct')
        commentSummaryItem['maxPage'] = data.get('maxPage')
        commentSummaryItem['score'] = data.get('score')
        commentSummaryItem['soType'] = data.get('soType')
        commentSummaryItem['imageListCount'] = data.get('imageListCount')
        yield commentSummaryItem

        for hotComment in data['hotCommentTagStatistics']:
            hotCommentTagItem = HotCommentTagItem()
            hotCommentTagItem['id'] = hotComment.get('id')
            hotCommentTagItem['name'] = hotComment.get('name')
            hotCommentTagItem['status'] = hotComment.get('status')
            hotCommentTagItem['rid'] = hotComment.get('rid')
            hotCommentTagItem['productId'] = hotComment.get('productId')
            hotCommentTagItem['count'] = hotComment.get('count')
            hotCommentTagItem['created'] = hotComment.get('created')
            hotCommentTagItem['modified'] = hotComment.get('modified')
            hotCommentTagItem['type'] = hotComment.get('type')
            hotCommentTagItem['canBeFiltered'] = hotComment.get('canBeFiltered')
            yield hotCommentTagItem

        for comment_item in data['comments']:
            comment = CommentItem()

            comment['id'] = comment_item.get('id')
            comment['productId'] = product_id
            comment['guid'] = comment_item.get('guid')
            comment['content'] = comment_item.get('content')
            comment['creationTime'] = comment_item.get('creationTime')
            comment['isTop'] = comment_item.get('isTop')
            comment['referenceID'] = comment_item.get('referenceId')
            comment['referenceName'] = comment_item.get('referenceName')
            comment['referenceType'] = comment_item.get('referenceType')
            comment['referenceTypeId'] = comment_item.get('referenceTypeId')
            comment['firstCategory'] = comment_item.get('firstCategory')
            comment['secondCategory'] = comment_item.get('secondCategory')
            comment['thirdCategory'] = comment_item.get('thirdCategory')
            comment['replyCount'] = comment_item.get('replyCount')
            comment['score'] = comment_item.get('score')
            comment['status'] = comment_item.get('status')
            comment['title'] = comment_item.get('title')
            comment['usefulVoteCount'] = comment_item.get('usefulVoteCount')
            comment['uselessVoteCount'] = comment_item.get('uselessVoteCount')
            comment['userImage'] = 'http://' + comment_item.get('userImage')
            comment['userImageUrl'] = 'http://' + comment_item.get('userImageUrl')
            comment['userLevelId'] = comment_item.get('userLevelId')
            comment['userProvince'] = comment_item.get('userProvince')
            comment['viewCount'] = comment_item.get('viewCount')
            comment['orderId'] = comment_item.get('orderId')
            comment['isReplyGrade'] = comment_item.get('isReplyGrade')
            comment['nickname'] = comment_item.get('nickname')
            comment['userClient'] = comment_item.get('userClient')
            comment['mergeOrderStatus'] = comment_item.get('mergeOrderStatus')
            comment['discussionId'] = comment_item.get('discussionId')

            # if(comment_item.get('discussionId')):
            #     comment['discussionId'] = comment_item.get('discussionId')
            # else:
            #     comment['discussionId'] =None
            comment['productColor'] = comment_item.get('productColor')
            comment['productSize'] = comment_item.get('productSize')
            comment['imageCount'] = comment_item.get('imageCount')
            comment['integral'] = comment_item.get('integral')
            comment['userImgFlag'] = comment_item.get('userImgFlag')
            comment['anonymousFlag'] = comment_item.get('anonymousFlag')
            comment['userLevelName'] = comment_item.get('userLevelName')
            comment['plusAvailable'] = comment_item.get('plusAvailable')
            comment['recommend'] = comment_item.get('recommend')
            comment['userLevelColor'] = comment_item.get('userLevelColor')
            comment['userClientShow'] = comment_item.get('userClientShow')
            comment['isMobile'] = comment_item.get('isMobile')
            comment['days'] = comment_item.get('days')
            comment['afterDays'] = comment_item.get('afterDays')
            yield comment

            if 'images' in comment_item:
                for image in comment_item['images']:
                    commentImageItem = CommentImageItem()
                    commentImageItem['_id'] = image.get('id')
                    commentImageItem['associateId'] = image.get('associateId')  # 和CommentItem的discussionId相同
                    commentImageItem['productId'] = image.get('productId')  # 不是ProductsItem的id，这个值为0
                    commentImageItem['imgUrl'] = 'http:' + image.get('imgUrl')
                    commentImageItem['available'] = image.get('available')
                    commentImageItem['pin'] = image.get('pin')
                    commentImageItem['dealt'] = image.get('dealt')
                    commentImageItem['imgTitle'] = image.get('imgTitle')
                    commentImageItem['isMain'] = image.get('isMain')
                    yield commentImageItem

            max_page = int(data.get('maxPage', '1'))
            if max_page > 60:
                max_page = 60
            for i in range(1, max_page):
                url = comment_url % (product_id, str(i))
                meta = dict()
                meta['product_id'] = product_id
                yield Request(url=url, callback=self.parse_comments2, meta=meta)

    def parse_comments2(self, response):

        """获取商品comment"""
        try:
            data = json.loads(response.text)
        except Exception as e:
            print('get comment failed:', e)
            return None

        product_id = response.meta['product_id']
        try:
            commentSummaryItem = CommentSummaryItem()
            commentSummary = data.get('productCommentSummary')
            commentSummaryItem['goodRateShow'] = commentSummary.get('goodRateShow')
            commentSummaryItem['poorRateShow'] = commentSummary.get('poorRateShow')
            commentSummaryItem['poorCountStr'] = commentSummary.get('poorCountStr')
            commentSummaryItem['averageScore'] = commentSummary.get('averageScore')
            commentSummaryItem['generalCountStr'] = commentSummary.get('generalCountStr')
            commentSummaryItem['showCount'] = commentSummary.get('showCount')
            commentSummaryItem['showCountStr'] = commentSummary.get('showCountStr')
            commentSummaryItem['goodCount'] = commentSummary.get('goodCount')
            commentSummaryItem['generalRate'] = commentSummary.get('generalRate')
            commentSummaryItem['generalCount'] = commentSummary.get('generalCount')
            commentSummaryItem['skuId'] = commentSummary.get('skuId')
            commentSummaryItem['goodCountStr'] = commentSummary.get('goodCountStr')
            commentSummaryItem['poorRate'] = commentSummary.get('poorRate')
            commentSummaryItem['afterCount'] = commentSummary.get('afterCount')
            commentSummaryItem['goodRateStyle'] = commentSummary.get('goodRateStyle')
            commentSummaryItem['poorCount'] = commentSummary.get('poorCount')
            commentSummaryItem['skuIds'] = commentSummary.get('skuIds')
            commentSummaryItem['poorRateStyle'] = commentSummary.get('poorRateStyle')
            commentSummaryItem['generalRateStyle'] = commentSummary.get('generalRateStyle')
            commentSummaryItem['commentCountStr'] = commentSummary.get('commentCountStr')
            commentSummaryItem['commentCount'] = commentSummary.get('commentCount')
            commentSummaryItem['productId'] = commentSummary.get('productId')  # 同ProductsItem的id相同
            commentSummaryItem['id'] = commentSummary.get('productId')
            commentSummaryItem['afterCountStr'] = commentSummary.get('afterCountStr')
            commentSummaryItem['goodRate'] = commentSummary.get('goodRate')
            commentSummaryItem['generalRateShow'] = commentSummary.get('generalRateShow')
            commentSummaryItem['jwotestProduct'] = data.get('jwotestProduct')
            commentSummaryItem['maxPage'] = data.get('maxPage')
            commentSummaryItem['score'] = data.get('score')
            commentSummaryItem['soType'] = data.get('soType')
            commentSummaryItem['imageListCount'] = data.get('imageListCount')
            yield commentSummaryItem

            for hotComment in data['hotCommentTagStatistics']:
                hotCommentTagItem = HotCommentTagItem()
                hotCommentTagItem['id'] = hotComment.get('id')
                hotCommentTagItem['name'] = hotComment.get('name')
                hotCommentTagItem['status'] = hotComment.get('status')
                hotCommentTagItem['rid'] = hotComment.get('rid')
                hotCommentTagItem['productId'] = hotComment.get('productId')
                hotCommentTagItem['count'] = hotComment.get('count')
                hotCommentTagItem['created'] = hotComment.get('created')
                hotCommentTagItem['modified'] = hotComment.get('modified')
                hotCommentTagItem['type'] = hotComment.get('type')
                hotCommentTagItem['canBeFiltered'] = hotComment.get('canBeFiltered')
                yield hotCommentTagItem

            for comment_item in data['comments']:
                comment = CommentItem()
                comment['id'] = comment_item.get('id')
                comment['productId'] = product_id
                comment['guid'] = comment_item.get('guid')
                comment['content'] = comment_item.get('content')
                comment['creationTime'] = comment_item.get('creationTime')
                comment['isTop'] = comment_item.get('isTop')
                comment['referenceId'] = comment_item.get('referenceId')
                comment['referenceName'] = comment_item.get('referenceName')
                comment['referenceType'] = comment_item.get('referenceType')
                comment['referenceTypeId'] = comment_item.get('referenceTypeId')
                comment['firstCategory'] = comment_item.get('firstCategory')
                comment['secondCategory'] = comment_item.get('secondCategory')
                comment['thirdCategory'] = comment_item.get('thirdCategory')
                comment['replyCount'] = comment_item.get('replyCount')
                comment['score'] = comment_item.get('score')
                comment['status'] = comment_item.get('status')
                comment['title'] = comment_item.get('title')
                comment['usefulVoteCount'] = comment_item.get('usefulVoteCount')
                comment['uselessVoteCount'] = comment_item.get('uselessVoteCount')
                comment['userImage'] = 'http://' + comment_item.get('userImage')
                comment['userImageUrl'] = 'http://' + comment_item.get('userImageUrl')
                comment['userLevelId'] = comment_item.get('userLevelId')
                comment['userProvince'] = comment_item.get('userProvince')
                comment['viewCount'] = comment_item.get('viewCount')
                comment['orderId'] = comment_item.get('orderId')
                comment['isReplyGrade'] = comment_item.get('isReplyGrade')
                comment['nickname'] = comment_item.get('nickname')
                comment['userClient'] = comment_item.get('userClient')
                comment['mergeOrderStatus'] = comment_item.get('mergeOrderStatus')
                comment['discussionId'] = comment_item.get('discussionId')

                comment['productColor'] = comment_item.get('productColor')
                comment['productSize'] = comment_item.get('productSize')
                comment['imageCount'] = comment_item.get('imageCount')
                comment['integral'] = comment_item.get('integral')
                comment['userImgFlag'] = comment_item.get('userImgFlag')
                comment['anonymousFlag'] = comment_item.get('anonymousFlag')
                comment['userLevelName'] = comment_item.get('userLevelName')
                comment['plusAvailable'] = comment_item.get('plusAvailable')
                comment['recommend'] = comment_item.get('recommend')
                comment['userLevelColor'] = comment_item.get('userLevelColor')
                comment['userClientShow'] = comment_item.get('userClientShow')
                comment['isMobile'] = comment_item.get('isMobile')
                comment['days'] = comment_item.get('days')
                comment['afterDays'] = comment_item.get('afterDays')
                yield comment

                if 'images' in comment_item:
                    for image in comment_item['images']:
                        commentImageItem = CommentImageItem()
                        commentImageItem['id'] = image.get('id')
                        commentImageItem['associateId'] = image.get('associateId')  # 和CommentItem的discussionId相同
                        commentImageItem['productId'] = image.get('productId')  # 不是ProductsItem的id，这个值为0
                        commentImageItem['imgUrl'] = 'http:' + image.get('imgUrl')
                        commentImageItem['available'] = image.get('available')
                        commentImageItem['pin'] = image.get('pin')
                        commentImageItem['dealt'] = image.get('dealt')
                        commentImageItem['imgTitle'] = image.get('imgTitle')
                        commentImageItem['isMain'] = image.get('isMain')
                        yield commentImageItem
        except Exception as e:
            print (e)